#
# catalog_manager.py 
#
# eCataloger base application for managing resource collections.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright 2009,2010- Jeremy Nelson, Tomichi Informatics LLC
#
# Import statements
#
import cgi,logging,sys,random
import datetime,os,re
import wsgiref.handlers

from google.appengine.ext import db
from google.appengine.api import users
from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.ext.webapp.util import login_required
from google.appengine.api import memcache
from django.utils import simplejson
import common.templatefilters

webapp.template.register_template_library('common.templatefilters')

import FRBRData,MODSData,basedata,PREMISData,FEDData
import ApplicationAndHelpData,SearchData

is_proxy = False
# Set-up environment to import zipped libraries
##sys.path.insert(0,'google.zip')
##from gdata import service
##from waveapi import robot
##sys.path.insert(0,'suds.zip')
##import suds.client

def get_entities(module_name):
    output = list()
    for k,v in module_name.__dict__.iteritems():
        if type(v) == db.PropertiedClass and k != 'BaseModel':
            entity = {'value':k,
                      'name':'%s (%s)' % (k,module_name.__name__)}
            existing_query = db.GqlQuery('SELECT * FROM %s' % k)
            entity['existing'] = existing_query.fetch(1000)
            output.append(entity)
    return output        

def GetEntityProperties(entity_name):
    entity_name = str(entity_name)
    raw_properties,class_properties = None,list()
    if hasattr(FRBRData,entity_name):
        raw_properties = getattr(FRBRData,entity_name)._properties
    elif hasattr(MODSData,entity_name):
        raw_properties = getattr(MODSData,entity_name)._properties
    elif hasattr(PREMISData,entity_name):
        raw_properties = getattr(PREMISData,entity_name)._properties
    elif hasattr(FEDData,entity_name):
        raw_properties = getattr(FEDData,entity_name)._properties
    for name,value in raw_properties.iteritems():
        save_record = False
        if hasattr(value,'required'):
            is_required = getattr(value,'required')
        record = {'label':name.replace("_"," ").title(),
                  'name':name,
                  'id':'id_%s' % name}            
        if type(value) == db.StringProperty:
            save_record = True
            record['type_of'] = 'text'
            if hasattr(value,'choices'):
                raw_choices = getattr(value,'choices')
                choices = list()
                if raw_choices:
                    for choice in raw_choices:
                        choices_dict = dict()
                        choices_dict['key'] = choice
                        choices_dict['label'] = choice.title()
                        choices.append(choices_dict)
                    record['choices'] = choices
            if is_required:
                record['is_required'] = True
        elif type(value) == db.StringListProperty:
            save_record = True
            record['type_of'] = 'select'
            if hasattr(value,'choices'):
                raw_choices = getattr(value,'choices')
                choices = list()
                if raw_choices:
                    for choice in raw_choices:
                        choices_dict = dict()
                        choices_dict['key'] = choice
                        choices_dict['label'] = choice.title()
                        choices.append(choices_dict)
                    record['choices'] = choices
        elif type(value) == db.BooleanProperty:
            save_record = True
            record['type_of'] = 'boolean'
        elif type(value) == db.ListProperty:
            save_record = True
            record['type_of'] = 'lkup_entities'
        elif type(value) == db.ReferenceProperty or \
             type(value) == db.SelfReferenceProperty:
            save_record = True
            record['type_of'] = 'lkup_entity'
        elif type(value) == basedata.BaseDateProperty:
            save_record = True
            record['type_of'] = 'date'
        elif type(value) == db.TextProperty:
            save_record = True
            record['type_of'] = 'textarea'
        elif type(value) == db.LinkProperty:
            save_record = True
            record['type_of'] = 'text'
        elif type(value) == db.PostalAddressProperty:
            save_record = True
            record['type_of'] = 'text'
        elif type(value) == PREMISData.agentIdentifierProperty or\
             type(value) == PREMISData.eventIdentifierProperty or\
             type(value) == PREMISData.objectIdentifierProperty or\
             type(value) == PREMISData.rightsStatementIdentifierProperty or\
             type(value) == PREMISData.AgentRolesProperty or\
             type(value) == PREMISData.ObjectRolesListProperty or\
             type(value) == PREMISData.ObjectIdentifierListProperty:
            save_record = True
            if hasattr(value.data_type,'types'):
                logging.error("ERROR %s has %s" % (value,value.data_type.types.keys()))
                record['type_of'] = 'select'
                choices = list()
                for choice in value.data_type.types.keys():
                    if choice:
                        choices_dict = dict()
                        choices_dict['key'] = choice
                        choices_dict['label'] = choice.title()
                        choices.append(choices_dict)
                record['choices'] = choices
            else:
                record['type_of'] = 'text'
        if save_record:
            class_properties.append(record)
    return sorted(class_properties,key=lambda x: x['name'])

#
# AJAX functionality for Application
#
class jQueryHandler(webapp.RequestHandler):
    ''' Handles jQuery AJAX Requests.'''
    def __init__(self):
        self.methods = jQueryMethods()
        
    def get(self):
        logging.error("In jQuery AJAX GET")
        self.error(403)

    def post(self):
        user_ip = self.request.remote_addr
        app_info_query = ApplicationAndHelpData.AppInfo.all()
        if app_info_query.count() > 0:
            app_info = app_info_query[0]
        else:
            app_info = None
            is_proxy = False
        if app_info:
            if user_ip >= app_info.start_ip_addr and\
               user_ip <= app_info.end_ip_addr:
                is_proxy = False
            else:
                is_proxy = True        
        function = self.request.get('func')
        args = self.request.get('args')
        if function[0] == '_':
            self.error(403)
            return
        func = getattr(self.methods,function,None)
        if not func:
            self.error(404)
            return
        result = func(args)
        self.response.out.write(simplejson.dumps(result))

class jQueryMethods:
    ''' Defines jQuery Methods '''
    def getEndeavorsByTitles(self,*args):
        next_letter = args[0]
        output = list()
        query = db.GqlQuery("SELECT * FROM titleInfo WHERE title >=:1 AND title <:2",
                            next_letter.lower(),
                            u"%s\ufffd" % next_letter.lower())
        results = query.fetch(100)
        for row in results:
            resource = None
            website_query = db.GqlQuery("SELECT * FROM WebSiteEndeavor WHERE title=:1",
                                        row)
            website = website_query.get()
            purchased_query = db.GqlQuery("SELECT * FROM PurchasedElectronicResourceEndeavor WHERE "\
                                          "title=:1",row)
            purchased_db = purchased_query.get()
            if website and purchased_db:
                pass
            elif website and not purchased_db:
                resource = website
            elif not website and purchased_db:
                resource = purchased_db
            else:
                pass
            if resource:
                result = {'title':row.display_form,
                          'description':'',
                          'key':str(resource.key())}
                if is_proxy and resource.manifestation.proxy_address:
                    result['url'] = resource.manifestation.proxy_address.value
                else:
                    result['url'] = resource.manifestation.access_address.value
                for note in resource.manifestation.get_notes():
                    if note.type == 'description':
                        result['description'] += note.value
                output.append(result)
        return sorted(output,key=lambda x: x['title'])

    def getEndeavorsBySubject(self,*args):
        subject = args[0]
        output = list()
        subject_query = db.GqlQuery("SELECT __key__ FROM subject WHERE topics=:1",
                                    subject.lower())
        subject_key = subject_query.get()
        if not subject_key:
            return 'ERROR: Subject %s not found' % subject
        concept_query = db.GqlQuery("SELECT __key__ FROM Concept WHERE subject=:1",
                                    subject_key)
        concept_key = concept_query.get()
        if not concept_key:
            return 'ERROR: Concept for subject %s (key=%s) not found' %\
                   (subject,subject_key)
        work_query = db.GqlQuery(\
            "SELECT __key__ FROM IndividualWork WHERE subjects=:1",
            concept_key)
        individual_works = work_query.fetch(1000)
        for row in individual_works:
            resource = None
            website_query = db.GqlQuery("SELECT * FROM WebSiteEndeavor WHERE work=:1",
                                        row)
            website = website_query.get()
            purchased_query = db.GqlQuery("SELECT * FROM PurchasedElectronicResourceEndeavor WHERE "\
                                          "work=:1",row)
            purchased_db = purchased_query.get()
            if website and purchased_db:
                pass
            elif website and not purchased_db:
                resource = website
            elif not website and purchased_db:
                resource = purchased_db
            else:
                pass
            if resource:
                result = {'title':resource.title.display_form,
                          'description':'',
                          'key':str(resource.key())}
                if is_proxy and resource.manifestation.proxy_address:
                    result['url'] = resource.manifestation.proxy_address.value
                else:
                    result['url'] = resource.manifestation.access_address.value
                for note in resource.manifestation.get_notes():
                    if note.type == 'description':
                        result['description'] += note.value
                output.append(result)
        return sorted(output,key=lambda x: x['title'])                
                
            
            
        

class RPCHandler(webapp.RequestHandler):
    '''Handles RPC Requests'''
    def __init__(self):
        webapp.RequestHandler.__init__(self)
        self.methods = RPCMethods()

    def get(self):
        logging.error("In RPCHandler GET")
        self.error(403)

    def post(self):
        # Sets global is_proxy variable depending on incoming request
        # IP address
        user_ip = self.request.remote_addr
        app_info_query = ApplicationAndHelpData.AppInfo.all()
        if app_info_query.count() > 0:
            app_info = app_info_query[0]
        else:
            app_info = None
            is_proxy = False
        if app_info:
            if user_ip >= app_info.start_ip_addr and\
               user_ip <= app_info.end_ip_addr:
                is_proxy = False
##            elif user_ip == '127.0.0.1':
##                cache_view = 'standard'
            else:
                is_proxy = True
                
        args = simplejson.loads(self.request.body)
        func, args = args[0], args[1:]

        # Access denied if client tries to call private/protected
        # function.
        if func[0] == '_':
            self.error(403)
            return

        # If function is not found return 404 msg to client
        func = getattr(self.methods,func, None)
        if not func:
            self.error(404)
            return

        result = func(*args)
        output = simplejson.dumps(result)
        self.response.out.write(output)


def getEntitiesFromKeys(key_list):
    ''' Supporting function for RPCMethods.getEntityList for recursive function
        call.'''
    results = list()
    for entity_key in key_list:
        output = list()
        logging.error("IN LIST value=%s type=%s" % (str(entity_key),type(entity_key)))
        if not entity_key:
            results.append(["None",])
        elif (type(entity_key) == str) or (type(entity_key)== unicode):
            results.append([str(entity_key),])
        else:
            a_entity = db.get(entity_key)
            output.append("Secondary item")
            for k,v in a_entity._properties.iteritems():
                prop_value = getattr(a_entity,k)
                if type(prop_value) == list:
                    results.append(getEntitiesFromKeys(prop_value))
                else:
                    try:
                        results.append(["<b>%s</b>: %s" % (str(k),
                                                          str(prop_value.key())),])
                    except:
                        results.append(["<b>%s</b>: %s"  % (str(k),
                                                           str(prop_value)),])
            # results.append(output)
    return results

def addEntityToDatastore(entity_dict):
    entity_name = entity_dict.pop('entity_type')
    if entity_dict.has_key('entity_key'):
        entity = db.get(entity_dict.pop('entity_key'))
    else:
        entity = None
    date_re = re.compile(r"(\w+)-*(\w+)*-*(\w+)*")
    none_re = re.compile(r"NONE",re.I)
    if hasattr(FRBRData,entity_name):
        entity_class = getattr(FRBRData,entity_name)
    elif hasattr(MODSData,entity_name):
        entity_class = getattr(MODSData,entity_name)
    elif hasattr(PREMISData,entity_name):
        entity_class = getattr(PREMISData,entity_name)
    elif hasattr(FEDData,entity_name):
        entity_class = getattr(FEDData,entity_name)
    params = {'created_by':users.get_current_user()}
    for k,v in entity_dict.iteritems():
        if hasattr(entity_class,k):
            entity_property = getattr(entity_class,k)
            if type(entity_property) == db.BooleanProperty:
                # Boolean values set to 0=false, 1=true in forms
                if entity:
                    setattr(entity,str(k),bool(int(v)))
                else:
                    params[str(k)] = bool(int(v))
            elif type(entity_property) == db.ReferenceProperty:
                if entity:
                    setattr(entity,str(k),db.Key(v))
                else:
                    params[str(k)] = db.Key(v)
            elif type(entity_property) == db.TextProperty:
                if entity:
                    setattr(entity,str(k),db.Text(v))
                else:
                    params[str(k)] = db.Text(v)
            elif type(entity_property) == db.StringListProperty:
                string_list = v.split("\n")
                if entity:
                    setattr(entity,str(k),[str(x) for x in string_list])
                else:
                    params[str(k)] = [str(x) for x in string_list]
            elif type(entity_property) == db.PostalAddressProperty:
                if entity:
                    setattr(entity,str(k),db.PostalAddress(v))
                else:
                    params[str(k)] = db.PostalAddress(v)
            elif type(entity_property) == db.RatingProperty:
                if entity:
                    setattr(entity,str(k),db.Rating(v))
                else:
                    params[str(k)] = db.Rating(v)
            elif type(entity_property) == basedata.BaseDateProperty:
                if none_re.search(v):
                    pass
                else:
                    date_result = date_re.search(v).groups()
                    date_value = basedata.BaseDate(year=int(date_result[0]))
                    if date_result[1]:
                        date_value.month = int(date_result[1])
                    if date_result[2]:
                        date_value.day = int(date_result[2])
                    if entity:
                        setattr(entity,str(k),date_value)
                    else:
                        params[str(k)] = date_value
            elif type(entity_property) == db.ListProperty:
                if type(v) != list:
                    key_list = [v]
                else:
                    key_list = v
                if entity:
                    setattr(entity,str(k),[db.Key(x) for x in key_list])
                else:
                    params[str(k)] = [db.Key(x) for x in key_list]
            else:
                if none_re.search(v):
                    pass
                elif entity:
                    setattr(entity,str(k),str(v))
                else:
                    params[str(k)] = str(v)               
    if entity:
        entity_key = entity.put()
    else:
        entity_key = entity_class(**params).put()
    return str(entity_key)

def getRecord(resource):
    ''' Supporting function for getEndeavorsByTitle and getEndeavorsByConcept
        RPC methods.'''
    record = {'key':str(resource.key()),
              'is_subscription':resource.manifestation.is_subscription,
              'display_form':resource.title.display_form,
              'description':''}
    if is_proxy and resource.manifestation.proxy_address:
        record['external_url'] = resource.manifestation.proxy_address.value
    else:
        record['external_url'] = resource.manifestation.access_address.value
    for note in resource.manifestation.get_notes():
        if note.type == 'description':
            record['description'] += note.value
    return record
    
class RPCMethods:
    '''Restricts RPC calls to methods defined in this class'''
    
    def addEntity(self,*args):
        entity_type = args[0]
        return self.getEntityForm(entity_type)

    def AddUsage(self,*args):
        endeavor_key = args[0]
        endeavor = db.get(db.Key(endeavor_key))
        endeavor.item.usage.append(datetime.datetime.today())
        endeavor.item.put()
        logging.error("Endeavor.item.usage is %s" % endeavor.item.usage)

    def checkUser(self,*args):
        user_info = args[0]
        output_dict = dict()
        output_dict['select_id'] = user_info[1]
        new_user = users.User(email=user_info[0])
        output_dict['user_email'] = new_user.email()
        rights_class = getattr(ApplicationAndHelpData,user_info[2])
        new_right = rights_class(created_by=users.get_current_user(),
                                 user=new_user)
        new_right.put()
        output_dict['user_id'] = new_user.user_id()
        return output_dict

    def addEndeavor(self,*args):
        ''' Returns endeavor form'''
        if not users.get_current_user():
            raise RuntimeError("USER must be logged for addEndeavor")
        endeavor_type = args[0]
        endeavor_class = getattr(FRBRData,endeavor_type)
        form_type = args[1]
        output = dict()
        if form_type == 'short':
            template_values = {'endeavor':endeavor_class}
            path = os.path.join(os.path.dirname(__file__),
                                'templates/quick_endeavor.html')
            output['short'] = template.render(path,template_values)
        else:
            endeavor_work = getattr(endeavor_class,'work')
            entity_type = str(endeavor_work.reference_class).split(".")[1][:-2]
            output['work'] = self.getEntityForm(entity_type)
            endeavor_expr = getattr(endeavor_class,'expression')
            expr_type = str(endeavor_expr.reference_class).split(".")[1][:-2]
            output['expression'] = self.getEntityForm(expr_type)
            endeavor_man = getattr(endeavor_class,'manifestation')
            man_type = str(endeavor_man.reference_class).split(".")[1][:-2]
            output['manifestation'] = self.getEntityForm(man_type)
        return output

    def clearCache(self):
        return memcache.flush_all()

    def displayReport(self,*args):
        report_type = args[0]
        element_value = args[1]
        if report_type == 'use':
            if element_value == 'all':
                entities = list()
                entities.extend(FRBRData.WebSiteEndeavor.all())
                entities.extend(FRBRData.PurchasedElectronicResourceEndeavor.all())
                entities.extend(ApplicationAndHelpData.HelpTopics.all())
                template_values = {'entities':sorted(entities,key=lambda x: x.display_form)}
            elif hasattr(FRBRData,element_value):
                frbr_class = getattr(FRBRData,element_value)
                frbr_entities = frbr_class.all()
                template_values = {'entities':frbr_entities}
            elif hasattr(ApplicationAndHelpData,element_value):
                help_class = getattr(ApplicationAndHelpData,element_value)
                help_topics = help_class.all()
                template_values = {'entities':help_topics}
            path = os.path.join(os.path.dirname(__file__),
                                'templates/usage_report.html')
        return template.render(path,template_values)

    def getCollection(self,*args):
        filter_type = args[0]
        resources = list()
        results = {'collection':[],
                   'filter_type':filter_type}
        from google.appengine.ext import db
        # Assume filter by first letter in title
        if len(filter_type) == 1:
            query = db.GqlQuery('SELECT * FROM PurchasedElectronicResourceEndeavor')
            databases = [endeavor for endeavor in query\
                         if endeavor.title.title.startswith(filter_type.lower())]
            for row in databases:
                resources.append(row)
            ws_query = db.GqlQuery("SELECT * FROM WebSiteEndeavor")
            websites = [endeavor for endeavor in ws_query\
                        if endeavor.title.title.startswith(filter_type.lower())]
            for ws in websites:
                resources.append(ws)
        # Assume filter by concept
        else:
            subject_query = db.GqlQuery("SELECT __key__ FROM subject WHERE topics=:1",
                                    filter_type.lower())
            subject_key = subject_query.get()
            query = db.GqlQuery("SELECT __key__ FROM Concept WHERE subject=:1",
                                subject_key)
            concept_key = query.get()
            if concept_key:
                db_query = db.GqlQuery("SELECT * FROM IndividualWork WHERE subjects=:1",
                                       concept_key)
                works = db_query.fetch(1000)
                for work in works:
                    purchased_query = db.GqlQuery("SELECT * FROM PurchasedElectronicResourceEndeavor WHERE work=:1",
                                                  work)
                    databases = purchased_query.fetch(1000)
                    for row in databases:
                        resources.append(row)
                    website_query = db.GqlQuery("SELECT * FROM WebSiteEndeavor WHERE work=:1",
                                                work)
                    websites = website_query.fetch(1000)
                    for ws in websites:
                        resources.append(ws)
        for resource in resources:
            record = {'title':resource.title.display_form,
                      'key':str(resource.key()),
                      'description':resource.manifestation.get_notes()[0].value,
                      'is_subscription':resource.manifestation.is_subscription}
            if is_proxy and resource.manifestation.proxy_address:
                record['url'] = resource.manifestation.proxy_address.value
            else:
                record['url'] = resource.manifestation.access_address.value
            results['collection'].append(record)
        results['collection'] = sorted(results['collection'],key=lambda x: x['title'])
        return results

                    
                

    def getEndeavor(self,*args):
        endeavor_key = args[0]
        endeavor = db.get(db.Key(endeavor_key))
        output = dict()
        output['titleInfo'] = {'title':endeavor.title.title,
                               'display_form':endeavor.title.display_form,
                               'key':str(endeavor.title.key())}
        if endeavor.title.subTitle:
            output['titleInfo']['subtitle'] = endeavor.title.subTitle
        output['concepts'] = []
        for concept_key in endeavor.work.subjects:
            concept = db.get(concept_key)
            output['concepts'].append({'display_form':concept.display_form,
                                       'concept_key':str(concept_key)})
        output['access_url'] = endeavor.manifestation.access_address.value
        output['access_url_key'] = str(endeavor.manifestation.access_address.key())
        if endeavor.manifestation.proxy_address:
            output['proxy_url'] = endeavor.manifestation.proxy_address.value
            output['proxy_url_key'] = str(endeavor.manifestation.proxy_address.key())
        output['publisher_name'] = endeavor.manifestation.publisher.display_form
        output['publisher_key'] = str(endeavor.manifestation.publisher.key())
        for note_key in endeavor.work.notes:
            note = db.get(note_key)
            if note.type == 'description':
                output['description'] = str(note.value)
                output['description_key'] = str(note_key)
        if endeavor.manifestation.is_subscription:
            output['is_subscription'] = 1
        if endeavor.manifestation.is_trial:
            output['is_trial'] = 1
        if endeavor.manifestation.popularity:
            output['popularity'] = str(endeavor.manifestation.popularity)
        return output

    def getEndeavorDisplay(self,*args):
        endeavor_key = args[0];
        endeavor = db.get(db.Key(endeavor_key))
        endeavor.item.usage.append(datetime.datetime.today())
        endeavor.item.put()
        endeavor_class = type(endeavor)
        output = self.getEndeavor(endeavor_key)
        publisher = endeavor.manifestation.publisher
        manifestation_class = type(endeavor.manifestation)
        all_manifestations = manifestation_class.all().filter("publisher =",
                                                              publisher)
        output['other_publisher_resources'] = []
        for manifestation in all_manifestations:
            all_endeavors = endeavor_class.all().filter("manifestation =",
                                                        manifestation)
            if all_endeavors:
                for row in all_endeavors:
                    if row.key() != endeavor.key():
                        result = {'endeavor_key':str(row.key()),
                                  'display_form':row.title.display_form}
                        output['other_publisher_resources'].append(result)
        return output

    def getEndeavorsByTitles(self,*args):
        output = {}
        purchased_resources = FRBRData.PurchasedElectronicResourceEndeavor.all()
        free_websites = FRBRData.WebSiteEndeavor.all()
        alpha_dict = {'a':[],'b':[],'c':[],'d':[],
                      'e':[],'f':[],'g':[],'h':[],
                      'i':[],'j':[],'k':[],'l':[],
                      'm':[],'n':[],'o':[],'p':[],
                      'q':[],'r':[],'s':[],'t':[],
                      'u':[],'v':[],'w':[],'w':[],
                      'y':[],'z':[]}
        for resource in purchased_resources:
            record = getRecord(resource)
            title_letter = resource.title.title[0].lower()
            if alpha_dict.has_key(title_letter):
                alpha_dict[title_letter].append(record)
            else:
                alpha_dict[title_letter] = [record,]
        for website in free_websites:
            record = getRecord(website)
            title_letter = website.title.title[0].lower()
            if alpha_dict.has_key(title_letter):
                alpha_dict[title_letter].append(record)
            else:
                alpha_dict[title_letter] = [record,]
        for k,v in alpha_dict.iteritems():
            alpha_dict[k] = sorted(v,key=lambda x: x['display_form'].lower())
        output['endeavors'] = sorted(alpha_dict.items(),
                                     key=lambda x: x[0])
        return output

    def getEndeavorsByConcepts(self,*args):
        purchased_resources = FRBRData.PurchasedElectronicResourceEndeavor.all()
        free_websites = FRBRData.WebSiteEndeavor.all()
        output = {}
        all_concepts = []
        concepts = FRBRData.Concept.all().order("display_form")
        for concept in concepts:
            concept_dict = {'display_form':concept.display_form,
                            'resources':[]}
            for resource in purchased_resources:
                record = getRecord(resource)
                for concept_key in resource.work.subjects:
                    if concept_key == concept.key():
                        concept_dict['resources'].append(record)
            for resource in free_websites:
                record = getRecord(resource)
                for concept_key in resource.work.subjects:
                    if concept_key == concept.key():
                        concept_dict['resources'].append(record)
            concept_list = sorted(concept_dict['resources'],
                                  key=lambda x: x['display_form'])
            concept_dict['resources'] = concept_list
            all_concepts.append(concept_dict)
        output['concepts'] = all_concepts
        return output
        
        

    def getEndeavors(self,*args):
        endeavor_class = args[0]
        query = db.GqlQuery("SELECT * FROM %s" % endeavor_class)
        endeavors = query.fetch(1000)
        output = list()
        for endeavor in endeavors:
            output.append({'key':str(endeavor.key()),
                           'display_form':endeavor.display_form})
        
        return output        
        
    def deleteEntity(self,*args):
        entity = db.get(args[0])
        if entity:
            entity.delete()
        return "Entity with key=%s removed from datastore" % args[0]

    def deleteUserRight(self,*args):
        user_right = db.get(args[0])
        if user_right:
            user_right.delete()
            return args[0]
        else:
            return None
        

    def getEntityList(self,*args):
        entity_name = args[0]
        query = db.GqlQuery('SELECT * FROM %s' % entity_name)
        entities = query.fetch(1000)
        results = list()
        for i,entity in enumerate(entities):
            record = [getattr(entity,
                             'display_label',
                             '%s %s' % (entity_name,i+1))]
            properties = list()
            for k,v in entity._properties.iteritems():
                prop_value = getattr(entity,k)
                if type(prop_value) == list:
#                    logging.error("IN GETENTITYLIST k=%s prop_value=%s" % (k,prop_value))
                    if len(prop_value) > 0:
                        properties.append(['%s' % str(k),getEntitiesFromKeys(prop_value)])
                    else:
                        properties.append(["<b>%s</b>: None" % str(k),])
##                    output_list = list()
##                    for value in prop_value:
##                        output_list.append([str(value),])
##                    properties.append(["<b>%s</b>: " % str(k),
##                                       output_list])
                else:
                    try:
                        properties.append(["<b>%s</b>: %s" % (str(k),
                                                         str(prop_value.key())),])
                    except:
                        properties.append(["<b>%s</b>: %s"  % (str(k),
                                                          str(prop_value)),])
            record.append(sorted(properties))
            results.append(record)
        return [entity_name,results]
    
    def getEntityListing(self,*args):
        entity_name = args[0]
        query = db.GqlQuery('SELECT * FROM %s' % entity_name)
        entities = query.fetch(1000)
        results = list()
        for entity in entities:
            record = {'key':str(entity.key())}
            for k,v in entity._properties.iteritems():
                prop_value = getattr(entity,k)
                if type(prop_value) == list:
                    record[str(k)] = [str(x) for x in prop_value]
                else:
                    record[str(k)] = str(prop_value)
            results.append(record)
        return results

    def getEntity(self,*args):
        entity_key = args[0]
        entity = db.get(entity_key)
        output = {'key':str(entity_key),
                  'properties':dict()}
        for k,v in entity._properties.iteritems():
            field = k
            field_value = getattr(entity,k)
            if field == 'key':
                logging.error("IS KEY=%s" % field_value)
                pass
            elif type(v) == db.StringListProperty:
                output['properties'][field] = list()
                for value in field_value:
                    output['properties'][field].append(str(value))
            elif type(v) == db.ListProperty:
                output['properties'][field] = list()
                for value in field_value:
                    output['properties'][field].append(self.getEntity(value))
            elif type(v) == db.ReferenceProperty or\
                 type(v) == db.SelfReferenceProperty:
                if field_value:
                    output['properties'][field] = self.getEntity(field_value.key())
            else:
                try:
                    output['properties'][field] = str(field_value.key())
                except:
                    output['properties'][field] = str(field_value)
##            elif type(field_value) == list:
##                field_values = list()
##                for value in field_value:
##                    try:
##                        valid_entity = db.get(field_value)
##                        field_value.append(self.getEntity(field_value))
##                    except:
##                        field_value.append(field_value)
            if field == 'display_label':
                output['display_label'] = field_value
        logging.error("ERROR ## GET ENTITY ## output=%s" % output)
        sorted(output['properties'])
        return output
                        
    def getEntityEditForm(self,*args):
        entity_key = args[0]
        entity = db.get(entity_key)
        entity_type = str(entity.__class__).split(".")[1][:-2]
        property_list = GetEntityProperties(entity_type)
        for row in property_list:
            if hasattr(entity,row['name']):
                row['value'] = getattr(entity,row['name'])
        template_values = {
            'mode':'edit',
            'entity_key':entity_key,
            'entity_type':entity_type,
            'entity_properties':property_list}
        path = os.path.join(os.path.dirname(__file__),
                            'templates/entity_form.html')
        return template.render(path,template_values)
        
        
    def getEntities(self,*args):
        #logging.error("ERROR in getEntities")
        entity_name = args[0]
        query = db.GqlQuery('SELECT * FROM %s' % entity_name)
        results = query.fetch(1000)
        output = {'type_of':entity_name,
                  'children':list()}
        for row in results:
            record = {'type_of':entity_name,
                      'properties':dict()}
            if hasattr(row,'display_label'):
                record['display_label'] = getattr(row,
                                                  'display_label')
            for k,v in row._properties.iteritems():
                field_value = getattr(row,k)
                #prop_value = v
                if k == 'key':
                    pass
                elif type(field_value) == list:
                    all_values = list()
                    for item in field_value:
                        #all_values.append(str(item))
                        try:
                            entity = db.get(item)
                            all_values.append(getEntity(item))
                        except:
                            all_values.append(str(item))
                    record['properties'][k] = all_values
                elif type(v) == db.ReferenceProperty or type(v) == db.SelfReferenceProperty:
                    if field_value:
                        logging.error("IS REFERENCEPROPERTY key=%s" % field_value.key())
                        record['properties'][k] = self.getEntity(field_value.key())
                else:
                    #record['properties'][k] = str(field_value)
                    try:
                        entity = db.get(field_value)
                        record['properties'][k] = self.getEntity(entity.key())
                    except:
                        record['properties'][k] = str(field_value)
            sorted(record['properties'])
            output['children'].append(record)
        #logging.error("ERROR #### output=%s" % output)
        return output

    def getHelpTopic(self,*args):
        help_topic = db.get(db.Key(args[0]))
        output = dict()
        if help_topic:
            help_topic.usage.append(datetime.datetime.today())
            help_topic.put()
            output['topic_key'] = str(help_topic.key())
            output['question'] = help_topic.question
            output['answer'] = help_topic.answer
        else:
            output = None
        return output
    
    def getWorks(self):
        query = db.GqlQuery('SELECT * FROM Work')
        works = query.fetch(1000)
        if not works:
            return None
        results = list()
        for work in works:
            record = dict(key=str(work.key()))
            if work.get_title():
                record['title'] = work.get_title()
            query2 = db.GqlQuery('SELECT __key__ FROM Expression WHERE work=:1',
                                 work)
            expressions = query2.fetch(1000)
            if expressions:
                record['expressions'] = [str(x) for x in expressions]
            results.append(record)
        return sorted(results,key=lambda x: x['title'])

    def initSetup(self,*args):
        output = args[0]
        init_email = output[0]
        is_admin = output[1]
        is_editor = output[2]
        is_contributor = output[3]
        app_name = output[4]
        institution_name = output[5]
        institution_addr = output[6]
        if init_email != users.get_current_user().email():
            user = users.User(email=init_email)
        else:
            user = users.get_current_user()
        if is_admin:
            admin = ApplicationAndHelpData.Administrators(user=user,
                                                         created_by=users.get_current_user())
            admin.put()
        if is_editor:
            editor = ApplicationAndHelpData.Editors(user=user,
                                                   created_by=users.get_current_user())
            editor.put()
        if is_contributor:
            query = ApplicationAndHelpData.Contributors.all().filter(\
                "user=",user)
            if not query.get():
                contributor = ApplicationAndHelpData.Contributors(user=user,
                                                                  created_by=users.get_current_user())
                contributor.put()
        corp_exists = FRBRData.CorporateBody.all().filter("name=",
                                                         institution_name).get()
        if not corp_exists:
            new_body = FRBRData.CorporateBody(created_by=users.get_current_user(),
                                              name=institution_name,
                                              display_form=institution_name,
                                              identification_status="provisional")
            if institution_addr:
                new_body.address = institution_addr
            new_body.put()
        else:
            new_body = corp_exists
        new_app_info = ApplicationAndHelpData.AppInfo(\
            created_by=users.get_current_user(),
            institution=new_body,
            app_name=app_name,
            display_form="%s AppInfo" % institution_name)
        new_app_info.put()
        return "%s" % new_app_info.key()

    def newEndeavor(self,*args):
        # Creates all new title, work, expression, manifestation, and item if
        # keys are not present.
        endeavor_info = args[0]
        user = users.get_current_user()
        endeavor_class = getattr(FRBRData,endeavor_info[0])
        title_key = endeavor_info[1][0]
        title_str = endeavor_info[1][1]
        subtitle = endeavor_info[1][2]
        title_display = endeavor_info[1][3]
        concept_keys = endeavor_info[2]
        access_address = endeavor_info[3]
        proxy_address = endeavor_info[4]
        publisher_key = endeavor_info[5]
        description = endeavor_info[6]
        is_subscription = int(endeavor_info[7])
        is_trial = int(endeavor_info[8])
        popularity = endeavor_info[9]
        endeavor_key = endeavor_info[10]
        language = endeavor_info[11]
        if title_key:
            title = db.get(db.Key(title_key))
            title.title = title_str
            title.display_label = title_display
            if not title.language:
                title.language = language
        else:
            title = MODSData.titleInfo(created_by=user,
                                       type="uniform",
                                       is_preferred_title=True,
                                       title=title_str,
                                       display_form=title_display,
                                       language=language)
        if subtitle:
            title.subtitle = subtitle
        title.put()
        if endeavor_key:
            work = db.get(db.Key(endeavor_key)).work
            if not work.titles.count(title.key()):
                work.titles.append(title.key())
            if not work.language:
                work.language = language
        else:
            work = FRBRData.IndividualWork(created_by=user,
                                           title=[title.key(),],
                                           language=language,
                                           display_form="%s IndividualWork" % title.display_form)
        work.subjects = []
        for concept_key in concept_keys:
            work.subjects.append(db.Key(concept_key))
        # If description exists, first position in array is the key
        if description[0]:
            description_entity = db.get(db.Key(description[0]))
            description_entity.value = db.Text(description[1])
            if not description_entity.language:
                description_entity.language = language
        else:
            description_entity = MODSData.note(created_by=user,
                                               type="description",
                                               value=db.Text(description[1]),
                                               display_form="%s Description" % title.display_form,
                                               language=language)
        description_entity.put()
        if not work.notes.count(description_entity.key()):
            work.notes.append(description_entity.key())
        if endeavor_key:
            expression = db.get(db.Key(endeavor_key)).expression
            if expression.title != title:
                expression.title = title
            if expression.work != work:
                expression.work = work
            if not expression.language:
                expression.language = language
        else:
            expression = FRBRData.SelfContainedExpression(\
                created_by=user,
                title=title,
                display_form="%s Expression" % title.display_form,
                language=language)
        if access_address[0]:
            access_url = db.get(db.Key(access_address[0]))
            access_url.value = access_address[1]
            if not access_url.language:
                access_url.language = language
        else:
            access_url = MODSData.url(created_by=user,
                                      value=access_address[1],
                                      display_form="%s URL" % title.display_form,
                                      language=language)
        access_url.put()
        if not expression.urls.count(access_url.key()):
            expression.urls.append(access_url.key())        
        if proxy_address[0]:
            proxy_url = db.get(db.Key(proxy_address[0]))
            proxy_url.value = proxy_address[1]
            if not proxy_url.language:
                proxy_url.language = language
            proxy_url.put()
        elif proxy_address[1]:
            proxy_url = MODSData.url(created_by=user,
                                     value=proxy_address[1],
                                     display_form="%s Proxy URL" % title.display_form,
                                     language=language)
            proxy_url.put()
        else:
            proxy_url = None
        if proxy_url:
            if not expression.urls.count(proxy_url.key()):
                expression.urls.append(proxy_url.key())
        expression.put()
        work.realized = expression
        work.put()
        expression.work = work
        expression.put()
        manifestation_class = endeavor_class.manifestation.reference_class
        if endeavor_key:
            manifestation = db.get(db.Key(endeavor_key)).manifestation
            if not manifestation.expressions.count(expression.key()):
                manifestation.expressions.append(expression.key())
            if not manifestation.language:
                manifestation.language = language
        else:
            manifestation = manifestation_class(created_by=user,
                                                title=title,
                                                expressions=[expression.key(),],
                                                digital_origin="born digital",
                                                display_form="%s Manifestation" %\
                                                title.display_form,
                                                mode_of_access="www",
                                                language=language)
        manifestation.access_address = access_url
        manifestation.publisher = db.get(db.Key(publisher_key))
        if not manifestation.notes.count(description_entity.key()):
            manifestation.notes.append(description_entity.key())        
        if proxy_url:
            manifestation.proxy_address = proxy_url
        if hasattr(manifestation,'is_subscription'):
            manifestation.is_subscription = bool(int(is_subscription))
        if hasattr(manifestation,'is_trial'):
            manifestation.is_trial = bool(int(is_trial))
        if popularity:
            manifestation.popularity = db.Rating(popularity)
        manifestation.put()
        if endeavor_key:
            item = db.get(db.Key(endeavor_key)).item
            if item.manifestation != manifestation:
                item.manifestation = manifestation
            if not item.language:
                item.language = language
        else:
            item = FRBRData.Item(created_by=user,
                                 manifestation=manifestation,
                                 display_form="%s Item" % title.display_form,
                                 language=language)
        item.put()
        if endeavor_key:
            endeavor = db.get(db.Key(endeavor_key))
            if endeavor.work != work:
                endeavor.work = work
            if endeavor.expression != expression:
                endeavor.expression = expression
            if endeavor.manifestation != manifestation:
                endeavor.manifestation = manifestation
            if endeavor.item != item:
                endeavor.item = item
            if endeavor.title != title:
                endeavor.title = title
            if not endeavor.language:
                endeavor.language = language
        else:
            endeavor = endeavor_class(created_by=user,
                                      work = work,
                                      expression = expression,
                                      manifestation = manifestation,
                                      item=item,
                                      title=title,
                                      display_form="%s Endeavor" % title.display_form,
                                      language=language)
        endeavor.put()
        return "Endeavor %s added with key of %s" %\
               (endeavor_class,endeavor.key())
        
        

    def populateHelpTopics(self,*args):
        topic_type = args[0]
        if topic_type == 'all' or not topic_type:
            query = db.GqlQuery("SELECT * FROM HelpTopics")
        else:
            query = db.GqlQuery("SELECT * FROM HelpTopics WHERE type_of=:1",
                                topic_type)
        help_topics = query.fetch(1000)
        path = os.path.join(os.path.dirname(__file__),
                            'templates/help_options.html')
        return template.render(path,{'help_topics':help_topics})

    def removeEndeavor(self,*args):
        endeavor_key = args[0]
        endeavor = db.get(db.Key(endeavor_key))
        work = endeavor.work
        for note_key in work.notes:
            note = db.get(note_key)
            db.delete(note)
        db.delete(work)
        db.delete(endeavor.expression)
        db.delete(endeavor.manifestation)
        db.delete(endeavor.item)
        db.delete(endeavor.title)
        db.delete(endeavor)
        return 'Removed Endeavor key=%s and associated entities '\
               'from datastore' % endeavor_key
        

    def saveAdmin(self,*args):
        admin_info = args[0]
        app_query = ApplicationAndHelpData.AppInfo.all()
        app_info = app_query.get()
        if app_info:
            if db.Key(admin_info['institution_key'])\
               != app_info.institution.key():
                app_info.institution = \
                                     db.get(db.Key(admin_info['institution_key']))
            if app_info.app_name != admin_info['app_name']:
                app_info.app_name = admin_info['app_name']
            if app_info.start_ip_addr != admin_info['start_ip']:
                app_info.start_ip_addr = admin_info['start_ip']
            if app_info.end_ip_addr != admin_info['end_ip']:
                app_info.end_ip_addr = admin_info['end_ip']
        else:
            app_info = ApplicationAndHelpData.AppInfo(created_by=users.get_current_user(),
                                                      institution=db.get(db.Key(admin_info['institution_key'])),
                                                      app_name=admin_info['app_name'],
                                                      start_ip_addr=admin_info['start_ip'],
                                                      end_ip_addr=admin_info['end_ip'])
        app_info.put()
        return "%s Administration information for %s saved with key=%s" %\
               (app_info.app_name,
                app_info.institution.display_form,
                app_info.key())
                                            

    def saveEntity(self,*args):
        entity_info = args[0]
        entity_dict = {}
        for row in entity_info:
            if row:
                if row[1] != 'NONE' or len(row[1].strip()) > 0:
                    if entity_dict.has_key(row[0]):
                        if type(entity_dict[row[0]]) == list:
                            entity_dict[row[0]].append(row[1])
                        else:
                            entity_dict[row[0]] = [entity_dict[row[0]],
                                                   row[1]]
                    else:
                        entity_dict[row[0]] = row[1]
        new_key = addEntityToDatastore(entity_dict)
        return new_key

    def saveHelp(self,*args):
        help_info = args[0]
        help_key = None
        if help_info.has_key('help_key'):
            help_key = help_info['help_key']
            if len(help_key) > 1:
                help_entity = db.get(help_key)
                help_entity.answer = db.Text(help_info['answer'])
                help_entity.question = help_info['question']
                help_entity.type_of=help_info['type_of']
            else:
                help_key = None
        if not help_key:
            help_entity = ApplicationAndHelpData.HelpTopics(created_by=users.get_current_user(),
                                                            answer=db.Text(help_info['answer']),
                                                            question=help_info['question'],
                                                            type_of=help_info['type_of'],
                                                            display_form="%s?" % help_info['question'].title())
        help_entity.put()
        help_output = {'message':'%s saved to HelpTopics with key=%s' %\
                       (help_info['question'],str(help_entity.key())),
                       'help_key':str(help_entity.key())}
        return help_output

    def search(self,*args):
        search_terms = args[0]
        query = SearchData.EndeavorSearch.all().search(search_terms)
        results = query.fetch(10)
        output = {'endeavors':[]}
        endeavor_keys = dict()
        for row in results:
            record = dict()
            endeavor = row.endeavor
            if endeavor_keys.has_key(str(endeavor.key())):
                pass
            else:
                record['key'] = str(endeavor.key())
                endeavor_keys[record['key']] = 0
                record['title'] = endeavor.title.display_form
                record['description'] = row.description
                if hasattr(endeavor.manifestation,'is_subscription'):
                    if endeavor.manifestation.is_subscription:
                        record['is_subscription'] = 'Yes'
                    else:
                        record['is_subscription'] = 'No'
                else:
                    record['is_subscription'] = 'No'
                if hasattr(endeavor.manifestation,'access_address'):
                    record['url'] = endeavor.manifestation.access_address.value
                else:
                    record['url'] = 'Not set'
                output['endeavors'].append(record)
        output['endeavors'] = sorted(output['endeavors'],key=lambda x: x['title'])       
        return output
        
        

    def getEntityForm(self,*args):
        entity_type = args[0]
        template_values = {
            'mode':'add',
            'entity_type':entity_type,
            'entity_properties':GetEntityProperties(entity_type)}
        path = os.path.join(os.path.dirname(__file__),
                            'templates/entity_form.html')
        return template.render(path,template_values)

    def getEndeavorClasses(self,*args):
        entity_name = args[0]
        entity_type = getattr(FRBRData,entity_name)
        output = {'work_class':str(entity_type.work.reference_class).split(".")[1][:-2],
                  'expression_class':str(entity_type.expression.reference_class).split(".")[1][:-2],
                  'manifestation_class':str(entity_type.manifestation.reference_class).split(".")[1][:-2]}
        return output
        
        

application = webapp.WSGIApplication([
    ('/rpc',RPCHandler)],debug=True)

def main():
    wsgiref.handlers.CGIHandler().run(application)

if __name__ == '__main__':
    main()
    
